import requests
import json
#import xpath
import datetime
import os, time, random
from MyOrm import Orm
from lxml import etree
from pymongo import MongoClient


class Sina():
    def __init__(self):
        doing = o.table(name='date_log', method='get', data={'status': 'doing'}, method2='all')
        if doing:
            self.roll_back(doing)
        client=MongoClient('47.98.203.142',27017)
        db=client.admin
        db = db.authenticate('myUserAdmin','abc123')
        self.db = client.stocknews

    def roll_back(self, data):
        """Dirty data rollback"""
        for item in data:
            o.table(name='data', method='delete', data={"date": item[1]})
            o.table(name='date_log', method='update', data=({'id':item[0]}, {"status": 'wait'}))
        print('rollback success')
    
    def dateRange(self, start, end, step=1, format="%Y-%m-%d"):
        strptime, strftime = datetime.datetime.strptime, datetime.datetime.strftime
        days = (strptime(end, format) - strptime(start, format)).days
        return [strftime(strptime(start, format) + datetime.timedelta(i), format) for i in range(0, days, step)]
    
    def put_task(self, date):
        for _ in date:
            o.table(name='date_log', method='put', data={'date': _, 'status': 'wait'})
    
    def deal_two(self, url):
        """deal journalism body"""
        """
        headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "max-age=0",
        "Connection": "keep-alive",
        "Host": "roll.news.sina.com.cn",
        "Upgrade-Insecure-Requests": 1,
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36"}
        """
        headers = {"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36"}
        response = requests.get(url=url, headers=headers).content
        try:
            response = response.decode('utf-8')
        except:
            response = response.decode('gbk')
        html = etree.HTML(response)
        texts= html.xpath('//*[@id="artibody"]/p/text()')
        if not texts:
            return "e"
        text = str()
        for t in texts:
            text += t + '\n'
        return text[:-1]
    
    def write_mongodb(self):
        index = 0
        while True:
            sql = "select id, date, url, title, kind, time from data limit %d,%d"%(index, 100)
            items = o.sql(sql)
            if len(items) == 0:
                return
            info_list = list()
            for _ in items:
                info = dict()
                info['id'] = _[0]
                info['date'] = _[1]
                info['url'] = _[2]
                info['title'] = _[3]
                info['kind'] = _[4]
                info['time'] = _[5]
                info['text'] = "NULL"
                info_list.append(info)
            index += 100
            self.db.sina.insert_many(info_list)
            print('schedule base: ', items[-1][0]," ", index)

    def update_mongodb(self):
        """save to mongodb"""
        index = 0
        while True:
            sql = "select id, text from data limit %d,%d"%(index, 100)
            items = o.sql(sql)
            if len(items) == 0:
                return
            for _ in items:
                self.db.sina.update_one({"id": _[0]},{"$set": {"text": _[1]}})
            index += 100
            print('schedule text: ', items[-1][0])
        
    def body_roll_back(self):
        """error save back"""
        sql = "SELECT * FROM data where length(text)<3;"
        items = o.sql(sql=sql, method='get')
        for item in items:
            o.table(name="data", method='update', data=({'id':item[0]}, {"text":"NULL"}))

    def get_body(self):
        """get journalism body"""
        self.body_roll_back()
        while True:
            item = o.table(name='data', method='get', data={'text': "NULL"}, method2='one')
            if not item:
                item = o.table(name='data', method='get', data={'text': "e"}, method2='one')
                if item:
                    self.body_roll_back()
                    continue
                else:
                    return self.update_mongodb()
            url = item[2]
            for _ in range(3):
                try: 
                    text = self.deal_two(url)
                except Exception as e: 
                    print('number of time is %d, error: %s'%(_, e))
                    text = "e"
                else:
                    break
            o.table(name='data', method='update', data=({"id":item[0]}, {"text": text}), method2='one')
            if text == 'e':
                print('schedule fail:', item)
                time.sleep(60*30)
            else:
                print('schedule success:', item)
    
    def deal_one(self, data):
        """deal json data"""
        data = data[data.index("{"):-1]
        data = eval(data, type('Dummy', (dict,), dict(__getitem__=lambda s,n:n))())
        return data

    def get_header(self, date, num=600):
        """journalism base data"""
        url = "http://roll.news.sina.com.cn/interface/rollnews_ch_out_interface.php?col=52&spec=&type=&date=%s&ch=03&k=&offset_page=0&offset_num=0&num=%d&asc=&page=1"%(date, num)
        
        headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
        "Accept-Encoding": "gzip, deflate",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "max-age=0",
        "Connection": "keep-alive",
        "Host": "roll.news.sina.com.cn",
        "Upgrade-Insecure-Requests":"1",
        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36"}
        for _ in range(3):
            try:
                response = requests.get(url, headers=headers).content
                try:
                    item = self.deal_one(response.decode('gbk'))
                except Exception as e:
                    print('decode: ', e)
                    item = self.deal_one(response.decode('utf-8'))
            except Exception as e:
                print('number of time is %d, error: %s'%(_, e))
                item=False
        if not item:
            with open('error/%s.html'%date, 'wb') as f:
                f.write(response)
            return False
        if item['count'] <= num:
            for _ in item['list']:
                info = dict()
                info['kind'] = _['channel']['title']
                info['title'] = _['title']
                info['url'] = _['url']
                info['date'] = date
                info['time'] = str(_['time'])
                info['text'] = "NULL"
                o.table(name="data", method="put", data=info)
            print(date, '完成')
            return True
        else: return self.get_header(date, num=item['count'])

    def main(self):
        while True:
            item = o.table(name='date_log', method='get', data={'status':'wait'}, method2='one')
            if not item:
                # get journalism body
                item = o.table(name='date_log', method='get', data={'status':'doing'}, method2='one')
                if not item:
                    return self.get_body()
                else:
                    Sina()
                    continue
            # get journalism base data
            o.table(name='date_log', method='update', data=({"id":item[0]}, {"status":"doing"}))
            # get result
            result = self.get_header(item[1])
            if result: o.table(name='date_log', method='update', data=({"date":item[1]}, {"status": 'end'}))
            #else: o.table(name='date_log', method='update', data=({"id":item[0]}, {"status": 'error'}))
            print('schedule: %s'%item[1])
            time.sleep(random.randint(2,3))

if __name__ == "__main__":
    # myself mysql-ORM
    o = Orm()
    s = Sina()
    #date = s.dateRange('2016-05-29', '2018-05-29')
    #s.put_task(date)
    #s.main()
    #s.get_body()
    s.write_mongodb()


